{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a52db231",
   "metadata": {},
   "source": [
    "## 异常点进一步分析\n",
    "\n",
    "获取异常点的业务意义在于可以部分异常点确实是经销商做的**非周期性商品促销**，识别异常点并纳入模型中，在预测时可以作为一种配置的信息以便协助模型更好的预测。\n",
    "\n",
    "关于异常点的处理：\n",
    "1. 由于数据点较少，且考虑到业务因素。这里只会去识别\"加性异常点\"，不考虑其他类型的异常点。\n",
    "2. 由于异常点数据上的随机性，因此不做进一步量化，识别出的点会以0-1的形式纳入模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9edfc3d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "import pickle\n",
    "import sys\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import numpy as np\n",
    "import pylab\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "plt.rcParams['axes.unicode_minus'] = False \n",
    "\n",
    "join = os.path.join\n",
    "abspath = os.path.abspath\n",
    "dirname = os.path.dirname\n",
    "\n",
    "CURRENT_PATH = os.getcwd()\n",
    "DATA_PATH = join(dirname(CURRENT_PATH) ,'DataAssets')\n",
    "SAVE_PATH = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b04bb82d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def thresholding_algo(y, lag, threshold, influence):\n",
    "    '''\n",
    "    \n",
    "    :param y:  array_like\n",
    "    :param lag: int\n",
    "    :param threshold: float\n",
    "    :param influence: float [0.0 - 1.0] 超出阈值的y 在平滑时的权重\n",
    "    :return dict: signals 是否超过阈值;avgFilter 均值平滑 ;stdFilter 方差平滑\n",
    "    '''\n",
    "    # 数据初始化\n",
    "    signals = np.zeros(len(y))\n",
    "    filteredY = np.array(y)\n",
    "    avgFilter = [0]*len(y)\n",
    "    stdFilter = [0]*len(y)\n",
    "\n",
    "    avgFilter[lag - 1] = np.mean(y[0:lag]) \n",
    "    stdFilter[lag - 1] = np.std(y[0:lag])\n",
    "    for i in range(lag, len(y)):\n",
    "        if abs(y[i] - avgFilter[i-1]) > threshold * stdFilter[i-1]: \n",
    "            # 超出阈值的处理\n",
    "            if y[i] > avgFilter[i-1]:\n",
    "                signals[i] = 1 \n",
    "            else:\n",
    "                signals[i] = -1 \n",
    "            filteredY[i] = influence * y[i] + (1 - influence) * filteredY[i-1]\n",
    "            avgFilter[i] = np.mean(filteredY[(i-lag+1):i+1])\n",
    "            stdFilter[i] = np.std(filteredY[(i-lag+1):i+1])\n",
    "        else: \n",
    "            signals[i] = 0\n",
    "            filteredY[i] = y[i]\n",
    "            avgFilter[i] = np.mean(filteredY[(i-lag+1):i+1])\n",
    "            stdFilter[i] = np.std(filteredY[(i-lag+1):i+1])\n",
    "\n",
    "    return dict(signals = np.asarray(signals),\n",
    "                avgFilter = np.asarray(avgFilter),\n",
    "                stdFilter = np.asarray(stdFilter))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f7ce2fb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "enog_data = pd.read_excel(join(DATA_PATH, 'cur_ts_melt.xlsx'))\n",
    "enog_data.columns = ['asc_code' , 'date' , 'val']\n",
    "enog_data = enog_data.sort_values(by = ['asc_code','date'],ascending=True)\n",
    "enog_data['date_str'] = enog_data['date'].apply(lambda x: dt.datetime.strftime(x , '%Y-%m-%d'))\n",
    "asc_code = enog_data['asc_code'].unique().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0fdd1a8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "abnormal_dict = {}\n",
    "one_asccode = asc_code[0]\n",
    "for one_asccode in asc_code:\n",
    "    enog_data_one = enog_data.loc[enog_data['asc_code'] == one_asccode]\n",
    "    if enog_data_one.shape[0] <= 12:\n",
    "        continue\n",
    "    val = enog_data_one['val'].tolist()\n",
    "    signals = thresholding_algo(val, 12, 3, 0.3)['signals']\n",
    "    signals = signals.tolist()\n",
    "    date_str = enog_data_one['date_str'].tolist()\n",
    "    signals_save = list(zip(date_str, signals))\n",
    "\n",
    "    abnormal_dict[one_asccode] = signals_save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fd0432c",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(join(SAVE_PATH,'abnormal_dict_.pickle') , 'wb') as f:\n",
    "    pickle.dump(abnormal_dict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb7c6202",
   "metadata": {},
   "outputs": [],
   "source": [
    "asccode = 0\n",
    "one_abnormal = abnormal_dict[asccode]\n",
    "one_abnormal_df = pd.DataFrame(one_abnormal, columns = ['date','val'])\n",
    "\n",
    "enog_data_one = enog_data[enog_data['asc_code'] == asccode]\n",
    "enog_data_one['date_str'] = enog_data_one['date'].apply(lambda x : dt.datetime.strftime(x , '%Y-%m-%d'))\n",
    "one_abnormal_df_final = one_abnormal_df.loc[(one_abnormal_df['date']>= '2020-01-01') & (one_abnormal_df['date']<= '2021-12-01')]\n",
    "enog_data_one_final = enog_data_one.loc[(enog_data_one['date_str'] >= '2020-01-01') & (enog_data_one['date_str'] <= '2021-12-01')]\n",
    "\n",
    "fig ,ax1 = plt.subplots(figsize=(10,8))\n",
    "ax1.plot(enog_data_one_final['date_str'] , enog_data_one_final['val'], label = '')\n",
    "for tick in ax1.get_xticklabels():\n",
    "    tick.set_rotation(45)\n",
    "ax2 = ax1.twinx()\n",
    "ax2.plot(one_abnormal_df_final['date'] , one_abnormal_df_final['val'], color = 'g',label = '')\n",
    "plt.title(f'asc_{asccode}_abnormal')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PyCharm (AccessoriesSales_pred_pj)",
   "language": "python",
   "name": "pycharm-9c735d0d"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}